package com.shujia.core

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object Demo8Union {
  def main(args: Array[String]): Unit = {
    //1、创建spark环境
    //创建配置文件对象
    val conf = new SparkConf()
    //指定spark执行默认，local：本地执行
    conf.setMaster("local")
    //spark 任务名
    conf.setAppName("wc")
    //创建spark上下文对象
    val sc = new SparkContext(conf)

    val rdd1: RDD[Int] = sc.parallelize(List(1, 2, 3, 4, 5, 6, 7))
    val rdd2: RDD[Int] = sc.parallelize(List(4, 5, 6, 6, 7, 8, 8))

    /**
     * union: 合并两个RDD,相当于sql中的union all
     */
    val unionRDD: RDD[Int] = rdd1.union(rdd2)

    //unionRDD.foreach(println)


    //去重重复数据
    //distinct: 会产生shuffle
    val distinctRDD: RDD[Int] = unionRDD.distinct()
    distinctRDD.foreach(println)
  }
}
